{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cbc8bf80",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def ProcessData_train(file):#处理所有训练数据，把str的标签转换为数字标签，存储在labeld中\n",
    "#     numlab = 0#变动的句子编号tag\n",
    "#     sentences=[]#存储所有单个句子\n",
    "#     labels=[]#数字标签\n",
    "#     numlabels=[]#记录句子属于对话的标号\n",
    "#     importants = []\n",
    "#     domains = []\n",
    "#     testdata = []\n",
    "#     role_train = []\n",
    "#     key_id =[]\n",
    "#     num = 0 \n",
    "#     flag = False\n",
    "#     train_select = []\n",
    "#     for key in file:\n",
    "#         if num%5!=1:#筛选测试数据\n",
    "#             train_select.append(key)\n",
    "#             dia = key['dialogue']\n",
    "#             for sen in dia:\n",
    "#                 flag = False\n",
    "#                 if sen['act']!='':\n",
    "#                     flag = True\n",
    "#                     if len(sen['text'])>maxlen:\n",
    "#                         sen['text'] = sen['text'][:maxlen-1]#截取长度为50字符\n",
    "#                     sentences.append(sen['text'])#将单个句子加入sentences\n",
    "#                     role_train.append(sen['speaker'])\n",
    "#                     labels.append (sen['act'])#将标签加入labels\n",
    "#                     numlabels.append(numlab)#记录这个句子所属于的对话\n",
    "#                     importants.append(sen['important']*1.0)\n",
    "#                     domains.append(dic[sen['domain']])\n",
    "                    \n",
    "#             if flag:\n",
    "#                 numlab+=1\n",
    "#         else:\n",
    "#             testdata.append(key)\n",
    "#         num+=1\n",
    "#     clabels = list(set(labels))#所有的labels去重\n",
    "#     clabels.sort( key=None, reverse=False)\n",
    "#     change_labels = clabels\n",
    "#     clabels = {val:key for key,val in enumerate(clabels)}#做一个字典出来，形式为{标签：标签对应数字编号}\n",
    "#     for key in range(len(labels)):#将所有的标签都换成数字标签便于计算\n",
    "#         lab = labels[key]\n",
    "#         labels[key]= clabels[lab]\n",
    "#     jj = json.dumps(train_select,ensure_ascii=False)\n",
    "#     f2 = open('train_data.json','w',encoding = 'utf-8')\n",
    "#     f2.write(jj)\n",
    "#     f2.close()\n",
    "#     return sentences,labels,numlabels,key_id,change_labels,clabels,testdata,role_train,importants,domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ac78cad",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_chane_ask = {\"套餐request_change\":\"套餐更改\",\"套餐inquire_service\":\"套餐咨询\",\"套餐request_cancel\":\"套餐取消\",\"业务inquire_service\":\"业务咨询\",\"业务request_change\":\"业务更改\",\"业务request_cancel\":\"业务取消\",\"流量request_change\":\"流量更改\",\"流量request_cancel\":\"流量包取消\",\"流量inquire_service\":\"流量咨询\",\"流量inform_fault\":\"信号故障\",\"宽带inform_fault\":\"宽带故障报修\",\"宽带inquire_service\":\"宽带业务咨询\",\"宽带request_cancel\":\"宽带业务取消\",\"账单inquire_service\":\"账单查询\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "326f19a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "dic = {-1:-1,'账单':0,\"流量\":1,\"套餐\":2,\"卡号\":3,\"业务\":4,\"宽带\":5,\"机顶盒\":6,\"工单\":7,\"活动\":8,\"其他\":9}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c0d9e64",
   "metadata": {},
   "outputs": [],
   "source": [
    "def ProcessData_train(file):#处理所有训练数据，把str的标签转换为数字标签，存储在labeld中\n",
    "    numlab = 0#变动的句子编号tag\n",
    "    sentences=[]#存储所有单个句子\n",
    "    labels=[]#数字标签\n",
    "    numlabels=[]#记录句子属于对话的标号\n",
    "    importants = []\n",
    "    domains = []\n",
    "    testdata = []\n",
    "    role_train = []\n",
    "    key_id =[]\n",
    "    num = 0 \n",
    "    flag = False\n",
    "    train_select = []\n",
    "    for key in file:\n",
    "        dia = key['dialogue']\n",
    "        for sen in dia:\n",
    "            flag = False\n",
    "            if sen['act']!='':\n",
    "                flag = True\n",
    "                if len(sen['text'])>maxlen:\n",
    "                    sen['text'] = sen['text'][:maxlen-1]#截取长度为50字符\n",
    "                sentences.append(sen['text'])#将单个句子加入sentences\n",
    "                role_train.append(sen['speaker'])\n",
    "                labels.append (sen['act'])#将标签加入labels\n",
    "                numlabels.append(numlab)#记录这个句子所属于的对话\n",
    "                importants.append(sen['important']*1.0)\n",
    "                domains.append(dic[sen['domain']])\n",
    "\n",
    "        if flag:\n",
    "            numlab+=1\n",
    "        num+=1\n",
    "    clabels = list(set(labels))#所有的labels去重\n",
    "    clabels.sort( key=None, reverse=False)\n",
    "    change_labels = clabels\n",
    "    clabels = {val:key for key,val in enumerate(clabels)}#做一个字典出来，形式为{标签：标签对应数字编号}\n",
    "    for key in range(len(labels)):#将所有的标签都换成数字标签便于计算\n",
    "        lab = labels[key]\n",
    "        labels[key]= clabels[lab]\n",
    "    return sentences,labels,numlabels,key_id,change_labels,clabels,testdata,role_train,importants,domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "00ff2787",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def ProcessData_test(file):#处理所有验证数据\n",
    "#     numlab = 0#变动的句子编号tag\n",
    "#     sentences=[]#存储所有单个句子\n",
    "#     labels=[]#数字标签\n",
    "#     numlabels=[]#记录句子属于对话的标号\n",
    "#     num_id = []\n",
    "#     speakers = []\n",
    "#     importants = []\n",
    "#     domains = []\n",
    "#     jj = json.dumps(file,ensure_ascii=False)\n",
    "#     f2 = open('test_data.json','w',encoding = 'utf-8')\n",
    "#     f2.write(jj)\n",
    "#     f2.close()\n",
    "#     for key in file:\n",
    "#         numlab = key['dialogue_idx']\n",
    "#         dia = key['dialogue']\n",
    "#         for sen in dia:\n",
    "#             if len(sen['text'])>maxlen:\n",
    "#                 sen['text'] = sen['text'][:maxlen-1]#截取长度为50的句子\n",
    "#             sentences.append(sen['text'])\n",
    "#             importants.append(sen['important'])\n",
    "#             domains.append(dic[sen['domain']])\n",
    "#             labels.append (sen['act'])\n",
    "#             speakers.append(sen['speaker'])\n",
    "#             numlabels.append(numlab)\n",
    "#             num_id.append(key['dialogue_idx'])\n",
    "#     return sentences,labels,numlabels,speakers,num_id,importants,domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4016abe7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def ProcessData_test(file):#处理所有验证数据\n",
    "    numlab = 0#变动的句子编号tag\n",
    "    sentences=[]#存储所有单个句子\n",
    "    labels=[]#数字标签\n",
    "    numlabels=[]#记录句子属于对话的标号\n",
    "    num_id = []\n",
    "    speakers = []\n",
    "    importants = []\n",
    "    domains = []\n",
    "    for key in file:\n",
    "        numlab = key['dialogue_idx']\n",
    "        dia = key['dialogue']\n",
    "        for sen in dia:\n",
    "            if len(sen['text'])>maxlen:\n",
    "                sen['text'] = sen['text'][:maxlen-1]#截取长度为50的句子\n",
    "            sentences.append(sen['text'])\n",
    "            importants.append(sen['important'])\n",
    "            domains.append(dic[sen['domain']])\n",
    "            labels.append (sen['act'])\n",
    "            speakers.append(sen['speaker'])\n",
    "            numlabels.append(numlab)\n",
    "            num_id.append(key['dialogue_idx'])\n",
    "    return sentences,labels,numlabels,speakers,num_id,importants,domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "80fb0bfa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def GetBatch_train(train_encodings,numlabels,labels,role_train,importants,domains):#因为后面要输入对话级别的数据，在这里做一个集合，把所有属于一个对话的句子放到一个数组中\n",
    "    length = len(train_encodings['input_ids'])\n",
    "    num_lab = 0\n",
    "    sens = []#单个对话的句子容器\n",
    "    labs = []#单个对话的标签容器\n",
    "    important = []\n",
    "    domain = []\n",
    "    speaker = []\n",
    "    batch_sentences = []#所有对话的句子容器\n",
    "    batch_labels = []#所有对话的标签容器\n",
    "    batch_speaker = []\n",
    "    batch_importants = []\n",
    "    batch_domains = []\n",
    "    for i in range(length):\n",
    "        if numlabels[i] == num_lab:#实现将所有同对话编号的句子加入进一个sens中，一个sens就是一个对话，将所有同对话编号的数字标签加入一个lab\n",
    "            sens.append(train_encodings['input_ids'][i])\n",
    "            important.append(importants[i])\n",
    "            domain.append(domains[i])\n",
    "            speaker.append(role_train[i])\n",
    "            labs.append(labels[i])\n",
    "        else:\n",
    "            batch_sentences.append(sens)\n",
    "            batch_labels.append(labs)\n",
    "            batch_speaker.append(speaker)\n",
    "            batch_importants.append(important)\n",
    "            batch_domains.append(domain)\n",
    "            sens = []\n",
    "            labs = []\n",
    "            speaker = []\n",
    "            num_lab+=1\n",
    "            domain = []\n",
    "            important = []\n",
    "            sens.append(train_encodings['input_ids'][i])\n",
    "            important.append(importants[i])\n",
    "            domain.append(domains[i])\n",
    "            speaker.append(role_train[i])\n",
    "            labs.append(labels[i])\n",
    "    return batch_sentences,batch_labels,batch_speaker,batch_importants,batch_domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ae5e3420",
   "metadata": {},
   "outputs": [],
   "source": [
    "def GetBatch_test(test_encodings,labels,sentences_test,speakers,importants,domains):#因为后面要输入对话级别的数据，在这里做一个集合，把所有属于一个对话的句子放到一个数组中\n",
    "    length = len(test_encodings['input_ids'])\n",
    "    num_lab = dia_id_test[0]\n",
    "    sens = []#单个对话的encoding句子容器\n",
    "    labs = []#单个对话的标签容器\n",
    "    sen = []#单个对话的原来句子容器\n",
    "    important = []\n",
    "    domain = []\n",
    "    batch_sen =[]#所有对话的原来句子容器\n",
    "    batch_sentences = []#所有对话的encoding句子容器\n",
    "    batch_labels = []#所有对话的标签容器\n",
    "    batch_speaker = []#存储说话者的身份\n",
    "    batch_ids = []#存储对话的ids\n",
    "    batch_importants = []\n",
    "    batch_domains = []\n",
    "    speaker = []\n",
    "    for i in range(length):\n",
    "        if dia_id_test[i] == num_lab:#实现将所有同对话编号的句子加入进一个sens中，一个sens就是一个对话，将所有同对话编号的数字标签加入一个lab\n",
    "            sens.append(test_encodings['input_ids'][i])\n",
    "            sen.append(sentences_test[i])\n",
    "            labs.append(labels[i])\n",
    "            speaker.append(speakers[i])\n",
    "            important.append(importants[i])\n",
    "            domain.append(domains[i])\n",
    "        else:\n",
    "            batch_ids.append(num_lab)\n",
    "            batch_sentences.append(sens)\n",
    "            batch_labels.append(labs)\n",
    "            batch_sen.append(sen)\n",
    "            batch_speaker.append(speaker)\n",
    "            batch_importants.append(important)\n",
    "            batch_domains.append(domain)\n",
    "            sens = []\n",
    "            labs = []\n",
    "            sen = []\n",
    "            speaker = []\n",
    "            domain = []\n",
    "            important = []\n",
    "            num_lab=dia_id_test[i]\n",
    "            sens.append(test_encodings['input_ids'][i])\n",
    "            sen.append(sentences_test[i])\n",
    "            labs.append(labels[i])\n",
    "            speaker.append(speakers[i])\n",
    "            important.append(importants[i])\n",
    "            domain.append(domains[i])\n",
    "    return batch_sentences,batch_labels,batch_sen,batch_speaker,batch_ids,batch_importants,batch_domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c2367f5e",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'torch' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_33116\\2950338946.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mclass\u001b[0m \u001b[0mDataset_test\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;31m#设置出dataset结构\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mencodings\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msentences\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mspeaker\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencodings\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mencodings\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlabels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msentences\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msentences\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'torch' is not defined"
     ]
    }
   ],
   "source": [
    "class Dataset_test(torch.utils.data.Dataset):#设置出dataset结构\n",
    "    def __init__(self, encodings,labels,sentences,speaker,importants,domains):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "        self.sentences = sentences\n",
    "        self.speaker = speaker\n",
    "        self.important = importants\n",
    "        self.domain = domains\n",
    "    def __getitem__(self, idx):\n",
    "        item = {'ids':torch.tensor(self.encodings[idx])}\n",
    "        item['labels']=self.labels[idx]\n",
    "        item['sentence'] = self.sentences[idx]\n",
    "        item['speaker'] = self.speaker[idx]\n",
    "        item['importants'] = torch.tensor(self.important[idx])\n",
    "        item['domains'] = torch.tensor(self.domain[idx])\n",
    "        return item\n",
    "    def __len__(self):\n",
    "        return len(self.encodings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0f2b1c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset_train(torch.utils.data.Dataset):#设置出dataset结构\n",
    "    def __init__(self, encodings,labels,speaker,importants,domains):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "        self.speaker = speaker\n",
    "        self.important = importants\n",
    "        self.domain = domains\n",
    "    def __getitem__(self, idx):\n",
    "        item = {'ids':torch.tensor(self.encodings[idx])}\n",
    "        item['labels']= torch.tensor((self.labels[idx]))\n",
    "        item['speaker'] = (self.speaker[idx])\n",
    "        item['importants'] = torch.tensor(self.important[idx])\n",
    "        item['domains'] = torch.tensor(self.domain[idx])\n",
    "        return item\n",
    "    def __len__(self):\n",
    "        return len(self.encodings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b429e2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset_domain(torch.utils.data.Dataset):#设置出dataset结构\n",
    "    def __init__(self, domains,ids):\n",
    "        self.domains = domains\n",
    "        self.ids= ids\n",
    "    def __getitem__(self, idx):\n",
    "        item = {'domains':torch.tensor(self.domains[idx])}\n",
    "        item['ids'] = torch.tensor(self.ids[idx])\n",
    "        return item\n",
    "    def __len__(self):\n",
    "        return len(self.domains)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5385338",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset_final(torch.utils.data.Dataset):#设置出dataset结构\n",
    "    def __init__(self,ids,intent,speaker):\n",
    "        self.intent = intent\n",
    "        self.ids= ids\n",
    "        self.speaker = speaker\n",
    "    def __getitem__(self, idx):\n",
    "        item = {'intent':self.intent[idx]}\n",
    "        item['speaker'] = self.speaker[idx]\n",
    "        item['ids'] = torch.tensor(self.ids[idx])\n",
    "        return item\n",
    "    def __len__(self):\n",
    "        return len(self.ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "04b7c926",
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (299462844.py, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"C:\\Users\\SunXuening\\AppData\\Local\\Temp\\ipykernel_35284\\299462844.py\"\u001b[1;36m, line \u001b[1;32m1\u001b[0m\n\u001b[1;33m    def get()\u001b[0m\n\u001b[1;37m             ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "def get_important_domain():\n",
    "    domain_train_ids = []\n",
    "    domain_important_train = []\n",
    "    for i in range(len(sentences_train)):\n",
    "        if importants_train[i]!=1:\n",
    "            a = torch.tensor([0.0 for i in range(10)])\n",
    "            domain_train_ids.append(train_encodings['input_ids'][i])\n",
    "            a[int(domains_train[i])]=1.0\n",
    "            domain_important_train.append(a)\n",
    "    return domain_train_ids,domain_important_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "754850d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def domain_idx(all_arr):\n",
    "    arr = [\"\" for i in range(10)]\n",
    "    for domain,idx in (dic.items()):\n",
    "        if idx!=-1:\n",
    "            arr[idx]=domain\n",
    "    domain_idx_change = arr\n",
    "    all_arr_domain =[]\n",
    "    for i in range(len(all_arr)):\n",
    "        arr_domain = []\n",
    "        for j in range(len(all_arr[i])):\n",
    "            arr_domain.append(domain_idx_change[all_arr[i][j]])\n",
    "        all_arr_domain.append(arr_domain)\n",
    "    return all_arr_domain"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "bert1",
   "language": "python",
   "name": "bert1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
